package com.demo.webmagic.pageProcessor;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.collections.functors.ConstantFactory;

import com.demo.common.constant.ConstantDtoFlight;
import com.demo.common.model.Proxyip;
import com.demo.common.model._MappingKit;
import com.demo.webmagic.dto.DtoProxyIp;
import com.demo.webmagic.pipeline.PipelineProxyIp;
import com.jfinal.kit.PropKit;
import com.jfinal.kit.StrKit;
import com.jfinal.log.Log;
import com.jfinal.plugin.activerecord.ActiveRecordPlugin;
import com.jfinal.plugin.druid.DruidPlugin;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.xsoup.Xsoup;

public class PageProcessorGoubanjia implements PageProcessor {

	private static final Log log = Log.getLog(PageProcessorGoubanjia.class);
	
	private Site site;
	
	@Override
	public Site getSite() {
		if(site == null) {
			site = Site.me().addStartUrl("http://proxy.goubanjia.com/free/gngn/index.shtml").setSleepTime(1000);
		}
		return site;
	}
	
	@Override
	public void process(Page page) {
		//System.out.println(page.getHtml());
		DtoProxyIp dtoProxyIp = new DtoProxyIp();
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		try {
			List<Proxyip> proxyIpList = new ArrayList<Proxyip>();
			List<Selectable> trList = page.getHtml().xpath("//table[@class='table']/tbody/tr").nodes();
			for (int i = 0; i < trList.size(); i++) {
				Proxyip proxyip = new Proxyip();
				Selectable tr = trList.get(i);
				//处理ip
				List<Selectable> ipList = tr.xpath("//td[@class='ip']/*[@style!='display: none;' AND @style!='display:none;']/text()").nodes();
				StringBuffer ipBuffer = new StringBuffer();
				for (Selectable selectable : ipList) {
					ipBuffer.append(selectable.get().trim());
				}
				proxyip.setIpStr(ipBuffer.toString().trim());
				List<Selectable> otherList = tr.xpath("//td[@class!='ip']").nodes();
				//if("高匿".equals(otherList.get(1).xpath("//a/text()").get().trim())) {
					proxyip.setPortStr(otherList.get(0).$("td", "text").toString().trim());
					proxyip.setLocation(otherList.get(4).xpath("//a[1]/text()").get() + " " + otherList.get(4).xpath("//a[2]/text()").get());
					proxyip.setValiDate(sdf.format(new Date()));
				//}
				proxyIpList.add(proxyip);
				/*List<Selectable> tdList = tr.xpath("//td/text()").nodes();
				if("高匿".equals(tdList.get(4).toString().trim()) && "HTTP".equals(tdList.get(3).toString().trim())) {
					Proxyip proxyip = new Proxyip();
					proxyip.setIpStr(tdList.get(0).toString().trim());
					proxyip.setPortStr(tdList.get(1).toString().trim());
					proxyip.setLocation(tdList.get(2).toString().trim());
					proxyip.setValiDate(tdList.get(5).toString().trim());
					proxyIpList.add(proxyip);
					for (Selectable td : tdList) {
						System.out.println(td.toString().trim());
					}
					System.out.println("+++++++++++++++++++");
				}*/
			}
			
			dtoProxyIp.setCode(ConstantDtoFlight.code_success);
			dtoProxyIp.setProxyIpList(proxyIpList);
		} catch(Exception e){
			log.error(e.getMessage());
			dtoProxyIp.setCode(ConstantDtoFlight.code_fail);
		} finally {
			page.putField("result", dtoProxyIp);
		}
	}
	
	public static void main(String[] args) {
		PropKit.use("a_little_config.txt");
		DruidPlugin dp = new DruidPlugin(PropKit.get("jdbcUrl"), PropKit.get("user"), PropKit.get("password"));
		ActiveRecordPlugin arp = new ActiveRecordPlugin(dp);
		_MappingKit.mapping(arp);
		
		dp.start();
		arp.start();
		//.addPipeline(new PipelineProxyIp())
		//String url = "http://proxy.goubanjia.com/free/gngn/index.shtml";
		try {
			Spider.create(new PageProcessorGoubanjia()).addPipeline(new PipelineProxyIp()).setDownloader(new SeleniumDownloader(URLDecoder.decode(PageProcessorAnxiaomo.class.getClass().getResource("/").getPath(), "utf-8") + "chromedriver.exe")).start();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
	}
}
